# Copyright (c) 2018 Presto Labs Pte. Ltd.
# Author: jaewon

import datetime
import getpass
import logging
import os
import subprocess
import sys
import tempfile

from absl import app, flags
import jinja2

from coin.base.datetime_util import iterate_date


class NoJobError(Exception):
  pass


def get_kr_group_parameter():
  return dict(group_name='kr',
              machines=['feed-01.ap-northeast-1.aws', 'feed-01.ap-northeast-2.aws'],
              market_type='Spot',
              exchanges=['Upbit', 'Bithumb'],
              symbols=['BTC-KRW,ETH-KRW,EOS-KRW,BCH-KRW,ETC-KRW'],
              api_override=['v1.snapshot'])


def get_okex_futures_group_parameter(v):
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.ap-northeast-2.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-01.ap-northeast-1.aws.huobi'
  ]
  symbols = [
      'BTC-USD.WEEK',
      'ETH-USD.WEEK',
      'EOS-USD.WEEK',
      'BCHN-USD.WEEK',
      'BTC-USD.QUARTER',
      'ETH-USD.QUARTER',
      'EOS-USD.QUARTER',
      'BCHN-USD.QUARTER'
  ]
  return dict(group_name='okex_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Okex'],
              symbols=symbols,
              api_override=['%s.realtime_move2bp' % v,
                            '%s.realtime' % v,
                            '%s.realtime_light_bbo' % v])


def get_okex_spot_group_parameter(v):
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-05.cn-hongkong.aliyun',
  ]
  return dict(group_name='okex_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Okex'],
              symbols=['BTC-USDT', 'ETH-USDT', 'EOS-USDT', 'BCHN-USDT'],
              api_override=['%s.snapshot' % v, '%s.realtime' % v])


def get_okex_futures_swap_group_parameter(v):
  machines = [
      'feed-05.cn-hongkong.aliyun',
      'feed-05.ap-northeast-1.aws',
  ]
  symbols = ['BTC-USD.PERPETUAL', 'ETH-USD.PERPETUAL', 'EOS-USD.PERPETUAL', 'LTC-USD.PERPETUAL']
  return dict(group_name='okex_futures_swap',
              machines=machines,
              market_type='Futures',
              exchanges=['Okex'],
              symbols=symbols,
              api_override=['%s-swap.realtime_move2bp' % v, '%s-swap.realtime' % v])


def get_huobi_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-01.ap-northeast-1.aws.huobi'
  ]
  symbols = [
      'BTC-USD.WEEK',
      'ETH-USD.WEEK',
      'EOS-USD.WEEK',
      'BCHABC-USD.WEEK',
      'BTC-USD.QUARTER',
      'ETH-USD.QUARTER',
      'EOS-USD.QUARTER',
      'BCHABC-USD.QUARTER'
  ]
  return dict(group_name='huobi_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Huobi'],
              symbols=symbols,
              api_override=['v1.bbo_move2bp', 'v1.incremental', 'v1.realtime_light_bbo'])


def get_huobi_futures_swap_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-01.ap-northeast-1.aws.huobi',
  ]
  symbols = ['BTC-USD.PERPETUAL', 'ETH-USD.PERPETUAL']
  return dict(group_name='huobi_futures_swap',
              machines=machines,
              market_type='Futures',
              exchanges=['Huobi'],
              symbols=symbols,
              api_override=['v1-swap.bbo', 'v1-swap.incremental', 'v1-swap.bbo_move2bp'])


def get_huobi_futures_linear_swap_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-01.ap-northeast-1.aws.huobi',
  ]
  symbols = ['BTC-USDT.PERPETUAL', 'ETH-USDT.PERPETUAL']
  return dict(group_name='huobi_futures_swap',
              machines=machines,
              market_type='Futures',
              exchanges=['Huobi'],
              symbols=symbols,
              api_override=['v1-linear-swap.bbo', 'v1-linear-swap.incremental',
                            'v1-linear-swap.bbo_move2bp'])


def get_huobi_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-01.ap-northeast-1.aws.huobi',
  ]
  return dict(group_name='huobi_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Huobi'],
              symbols=['BTC-USDT', 'ETH-USDT', 'EOS-USDT', 'XRP-USDT'],
              api_override=['v1.mbp5', 'v1.bbo'])


def get_binance_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.ap-northeast-2.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-01.ap-northeast-1.aws.huobi',
  ]
  symbols = [
      'BTC-USDT.PERPETUAL', 'ETH-USDT.PERPETUAL', 'EOS-USDT.PERPETUAL', 'BCHABC-USDT.PERPETUAL', 'XRP-USDT.PERPETUAL',
  ]
  return dict(group_name='binance_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Binance'],
              symbols=symbols,
              api_override=['v1.l1_realtime', 'v1.realtime_light_bbo', 'v1.l1_realtime_move2bp'])


def get_binance_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.ap-northeast-2.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-01.ap-northeast-1.aws.huobi'
  ]
  return dict(group_name='binance_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Binance'],
              symbols=['BTC-USDT', 'ETH-USDT', 'EOS-USDT', 'BCHABC-USDT', 'UST-USDT'],
              api_override=['v1.snapshot'])


def get_binanceus_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.us-east-1.aws',
  ]
  return dict(group_name='binanceus_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Binanceus'],
              symbols=["BUSD-USDT", "USDT-USD"],
              api_override=['v1.snapshot'])


def get_bitmex_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.ap-northeast-2.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-01.ap-northeast-1.aws.huobi'
  ]
  return dict(group_name='bitmex',
              machines=machines,
              market_type='Futures',
              exchanges=['Bitmex'],
              symbols=[
                  "BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL", "ADA-BTC.QUARTER", "BTC-USD.QUARTER",
                  "EOS-BTC.QUARTER", "ETH-BTC.QUARTER", "LTC-BTC.QUARTER", "TRX-BTC.QUARTER",
                  "XRP-BTC.QUARTER", "BCHN-USD.PERPETUAL", "BCHN-BTC.QUARTER", "XRP-USDT.PERPETUAL",
                  "BTC-EUR.QUARTER", "ETH-USDT.QUARTER", "BTC-USDT.PERPETUAL", "BTC-USDT.QUARTER",
                  "DOGE-USD.PERPETUAL", "BCHN-USDT.PERPETUAL", "LUNA-USD.PERPETUAL", "DOT-USD.PERPETUAL",
                  "DEFIMEX-USD.PERPETUAL", "SOL-USDT.PERPETUAL", "DOGE-USDT.PERPETUAL", "ADA-USD.PERPETUAL",
                  "EOS-USD.PERPETUAL", "LTC-USDT.PERPETUAL", "ETH-USDT.PERPETUAL", "AVAX-USD.PERPETUAL",
                  "BTC-EUR.PERPETUAL", "ALTMEX-USD.PERPETUAL", "LINK-USD.PERPETUAL", "SOL-USD.PERPETUAL",
                  "AXS-USD.PERPETUAL", "BNB-USD.PERPETUAL"],
              api_override=['v1.realtime', 'v1.realtime_light_bbo'])


def get_bitflyer_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitflyer',
              machines=machines,
              market_type='Futures',
              exchanges=['Bitflyer'],
              symbols=['BTC-JPY.PERPETUAL', 'BTC-JPY.IMMEDIATE'],
              api_override=['v1.realtime'])


def get_bitbank_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitbank',
              machines=machines,
              market_type='Spot',
              exchanges=['Bitbank'],
              symbols=['BTC-JPY', 'XRP-JPY', 'ETH-BTC'],
              api_override=['v1.realtime'])


def get_bitfinex_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.ap-northeast-2.aws',
      'feed-05.cn-hongkong.aliyun',
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-01.ap-northeast-1.aws.huobi'
  ]
  return dict(group_name='bitfinex',
              machines=machines,
              market_type='Spot',
              exchanges=['Bitfinex'],
              symbols=['BTC-USDT', 'ETH-USDT', 'EOS-USDT', 'BCH-USDT', 'ETC-USDT'],
              api_override=['v1.snapshot'])


def get_gdax_group_parameter():
  machines = [
      'feed-02.eu-west-1.aws',
      'feed-02.us-east-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='gdax',
              machines=machines,
              market_type='Spot',
              exchanges=['Gdax'],
              symbols=['BTC-USD', 'ETH-USD'],
              api_override=['v1.snapshot'])


def get_kraken_group_parameter():
  machines = [
      'feed-02.us-east-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='kraken',
              machines=machines,
              market_type='Spot',
              exchanges=['Kraken'],
              symbols=['BTC-USD', 'ETH-USD'],
              api_override=['v1.snapshot'])


def get_upbit_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='upbit',
              machines=machines,
              market_type='Spot',
              exchanges=['Upbit'],
              symbols=['BTC-KRW', 'ETH-KRW', 'XRP-KRW'],
              api_override=['v1.realtime'])


def get_coinone_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='coinone',
              machines=machines,
              market_type='Spot',
              exchanges=['Coinone'],
              symbols=[
                  'BTC-KRW',
                  'BCHABC-KRW',
                  'ETH-KRW',
                  'ETC-KRW',
                  'XRP-KRW',
                  'QTUM-KRW',
                  'LTC-KRW',
                  'IOTA-KRW',
                  'EOS-KRW',
                  'BCHSV-KRW'
              ],
              api_override=['v2.realtime'])


def get_quoinex_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='quoinex',
              machines=machines,
              market_type='Spot',
              exchanges=['Quoinex'],
              symbols=['BTC-JPY', 'ETH-JPY', 'XRP-JPY'],
              api_override=['v2.realtime'])


def get_bithumb_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(
      group_name='bithumb',
      machines=machines,
      market_type='Spot',
      exchanges=['Bithumb'],
      symbols=['BTC-KRW', 'ETH-KRW', 'BCHSV-KRW', 'XRP-KRW', 'EOS-KRW', 'BCHABC-KRW', 'LINK-KRW'],
      api_override=['v2.realtime'])


def get_gopax_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(
      group_name='gopax',
      machines=machines,
      market_type='Spot',
      exchanges=['Gopax'],
      symbols=['BTC-KRW', 'ETH-KRW', 'XRP-KRW', 'EOS-KRW', 'BCHN-KRW', 'LINK-KRW'],
      api_override=['v1.realtime'])


def get_bybit_spot_group_parameter():
  machines = ['feed-05.ap-northeast-1.aws', 'feed-05.cn-hongkong.aliyun']
  parameters = []
  parameters.append(dict(group_name='bybit',
              machines=machines,
              market_type='Spot',
              exchanges=['Bybit'],
              symbols=["ETH-USDT", "BTC-USDT"],
              api_override=['v1.realtime']))
  parameters.append(dict(group_name='bybit',
              machines=machines,
              market_type='Spot',
              exchanges=['Bybit'],
              symbols=["ETH-USDT", "BTC-USDT"],
              api_override=['v3.snapshot']))
  return parameters


def get_bybit_futures_group_parameter():
  machines = ['feed-05.ap-northeast-1.aws', 'feed-05.cn-hongkong.aliyun']
  parameters = []
  parameters.append(dict(group_name='bybit',
              machines=machines,
              market_type='Futures',
              exchanges=['Bybit'],
              symbols=['BTC-USD.PERPETUAL', 'ETH-USD.PERPETUAL'],
              api_override=['v3.realtime']))
  return parameters


def get_bybit_linear_group_parameter():
  machines = ['feed-05.ap-northeast-1.aws', 'feed-05.cn-hongkong.aliyun']
  parameters = []
  parameters.append(dict(group_name='bybit-linear',
              machines=machines,
              market_type='Futures',
              exchanges=['Bybit'],
              symbols=['BTC-USDT.PERPETUAL', "ETH-USDT.PERPETUAL"],
              api_override=['v3-linear.realtime']))
  parameters.append(dict(group_name='bybit-linear',
              machines=machines,
              market_type='Futures',
              exchanges=['Bybit'],
              symbols=['BTC-USDC.PERPETUAL', "ETH-USDC.PERPETUAL"],
              api_override=['v3-linear-usdc.realtime']))
  return parameters


def get_ftx_futures_group_parameter():
  machines = ['feed-05.ap-northeast-1.aws', ]
  return dict(group_name='ftx_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Ftx'],
              symbols=['BTC-USD.PERPETUAL', 'ETH-USD.PERPETUAL', 'BTC-USD.QUARTER'],
              api_override=['v1.realtime'])


def get_ftx_group_parameter():
  machines = ['feed-05.ap-northeast-1.aws', ]
  return dict(group_name='ftx',
              machines=machines,
              market_type='Spot',
              exchanges=['Ftx'],
              symbols=['BTC-USDT', 'ETH-USDT', 'BTC-USD', 'ETH-USD'],
              api_override=['v1.realtime'])


def get_bitstamp_spot_group_parameter():
  machines = [
      'feed-02.eu-west-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitstamp_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Bitstamp'],
              symbols=['BTC-USD', 'ETH-USD'],
              api_override=['v2.snapshot'])


def get_korbit_spot_group_parameter():
  machines = [
      'feed-02.ap-northeast-2.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='korbit_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Korbit'],
              symbols=['BTC-KRW', 'ETH-KRW'],
              api_override=['v1.snapshot'])


def get_bitbank_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitbank_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Bitbank'],
              symbols=['BTC-JPY', 'ETH-BTC', 'LTC-BTC'],
              api_override=['v1.realtime'])


def get_prex_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='prex_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Prex'],
              symbols=['BTC-USD.PERPETUAL', 'ETH-USD.PERPETUAL', 'LTC-USD.PERPETUAL'],
              api_override=['v1.realtime'])


def get_lmaxdigital_spot_group_parameter():
  machines = [
      'feed-02.eu-west-1.aws',
  ]
  return dict(group_name='lmaxdigital_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Lmaxdigital'],
              symbols=['BTC-USD', 'ETH-USD', 'LTC-USD'],
              api_override=['v1.realtime'])


def get_deribit_futures_group_parameter():
  machines = [
      'feed-02.eu-west-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='deribit_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Deribit'],
              symbols=['BTC-USD.PERPETUAL', 'BTC-USD.WEEK', 'ETH-USD.QUARTER'],
              api_override=['v2.realtime'])


def get_deribit_options_group_parameter(trading_date):
  machines = [
      'feed-02.eu-west-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]

  def find_three_option_files(symbol, options_dir):
    rtn_option_files = []
    for subdir, _, files in os.walk(options_dir):
      for file_name in files:
        if symbol in file_name and os.stat(os.path.join(subdir, file_name)).st_size > 51200:
          rtn_option_files.append(file_name.replace('.W0.lz4', ''))
        if len(rtn_option_files) >= 3:
          return rtn_option_files
    return rtn_option_files

  options_dir = "/remote/iosg/coin-2/feed_cache2/%s/%s/Options.Deribit.v2.realtime/"\
   % (machines[1], trading_date)
  
  deribit_options_symbols = find_three_option_files("BTC-USD", options_dir)
  deribit_options_symbols.extend(find_three_option_files("ETH-USD", options_dir))

  return dict(group_name='deribit_options',
              machines=machines,
              market_type='Options',
              exchanges=['Deribit'],
              symbols=deribit_options_symbols,
              api_override=['v2.realtime'])


def get_coincheck_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='coincheck_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Coincheck'],
              symbols=["BTC-JPY", "MONA-JPY"],
              api_override=['v1.realtime'])


def get_uniswap_spot_group_parameter():
  machines = [
      'feed-01.ap-southeast-1.aws',
  ]
  return dict(group_name='uniswap_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Uniswap'],
              symbols=["ETH-USDC.5bp", "ETH-USDC.30bp", "ETH-USDT.5bp", "ETH-BTC.5bp",
                       "USDC-USDT.1bp", "ETH-USDT.30bp", "ETH-DAI.5bp", "ETH-BTC.30bp",
                       "ETH-DAI.30bp", "USDC-DAI.1bp", "BTC-USDC.30bp", "USDC-USDT.5bp",
                       "USDC-DAI.5bp", "APE-ETH.30bp", "LINK-ETH.30bp", "UNI-ETH.30bp"],
              api_override=['v3.snapshot'])


def get_uniswapv3_arbitrum_spot_group_parameter():
  machines = [
      'feed-01.ap-southeast-1.aws',
  ]
  return dict(group_name='uniswapv3-arbitrum_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Uniswap'],
              symbols=["ETH-USDC.5bp", "ETH-USDC.30bp", "ETH-USDT.5bp", "ARB-USDC.5bp"],
              api_override=['v3-arbitrum.snapshot'])


def get_uniswapv2_spot_group_parameter():
  machines = [
      'feed-01.ap-southeast-1.aws',
  ]
  return dict(group_name='uniswapv2_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Uniswap'],
              symbols=["ETH-USDC", "ETH-USDT", "CCX-ETH"],
              api_override=['v2.snapshot'])


def get_pancakeswap_spot_group_parameter():
  machines = [
      'feed-01.ap-southeast-1.aws',
  ]
  return dict(group_name='pancakeswap_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Pancakeswap'],
              symbols=["BNB-BUSD", "BNB-USDT", "CAKE-BNB", "USDT-BUSD", "USDC-BUSD", "ETH-BNB",
                       "BTC-BNB", "BTC-BUSD", "ETH-BTC", "MBOX-BNB", "ETH-USDC", "CAKE-BUSD"],
              api_override=['v2.snapshot'])


def get_gateio_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='gateio_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Gateio'],
              symbols=["ETH-USDT", "BTC-USDT", "ETH-BTC", "FIL-ETH"],
              api_override=['v4.realtime'])


def get_gateio_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  parameters = []
  parameters.append(dict(group_name='gateio_futures',
                    machines=machines,
                    market_type='Futures',
                    exchanges=['Gateio'],
                    symbols=["ETH-USDT.PERPETUAL", "BTC-USDT.PERPETUAL"],
                    api_override=['v4.realtime']))
  parameters.append(dict(group_name='gateio_futures',
                    machines=machines,
                    market_type='Futures',
                    exchanges=['Gateio'],
                    symbols=["ETH-USD.PERPETUAL", "BTC-USD.PERPETUAL"],
                    api_override=['v4-btc.realtime']))
  parameters.append(dict(group_name='gateio_futures',
                    machines=machines,
                    market_type='Futures',
                    exchanges=['Gateio'],
                    symbols=["BTC-USD.QUARTER"],
                    api_override=['v4-delivery-btc.realtime']))
  parameters.append(dict(group_name='gateio_futures',
                    machines=machines,
                    market_type='Futures',
                    exchanges=['Gateio'],
                    symbols=["BTC-USDT.QUARTER"],
                    api_override=['v4-delivery.realtime']))
  return parameters


def get_phemex_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-05.cn-hongkong.aliyun',
  ]
  parameters = []
  parameters.append(dict(group_name='phemex_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Phemex'],
              symbols=["BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL", "XTZ-USD.PERPETUAL"],
              api_override=['v1.realtime']))
  parameters.append(dict(group_name='phemex_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Phemex'],
              symbols=["BTC-USDT.PERPETUAL", "ETH-USDT.PERPETUAL", "XTZ-USDT.PERPETUAL"],
              api_override=['v1-linear.realtime']))
  return parameters


def get_phemex_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-05.cn-hongkong.aliyun',
  ]
  return dict(group_name='phemex_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Phemex'],
              symbols=["BTC-USDT", "ETH-USDT", "XTZ-USDT"],
              api_override=['v1.realtime'])


def get_kucoin_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='kucoin_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Kucoin'],
              symbols=["BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL", "XTZ-USDT.PERPETUAL"],
              api_override=['v1.realtime'])


def get_kucoin_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='kucoin_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Kucoin'],
              symbols=["BTC-USDT", "ETH-USDT", "XTZ-USDT"],
              api_override=['v1.realtime'])


def get_dydx_futures_group_parameter():
  machines = [
      'feed-02.us-east-1.aws',
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='dydx_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Dydx'],
              symbols=["BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL", "LTC-USD.PERPETUAL"],
              api_override=['v3.realtime'])


def get_btcturk_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='btcturk_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Btcturk'],
              symbols=["BTC-USDT", "ETH-USDT", "BTC-TRY", "ETH-TRY"],
              api_override=['v1.realtime'])


def get_mexc_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='mexc_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Mexc'],
              symbols=["BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL",
                       "BTC-USDT.PERPETUAL", "ETH-USDT.PERPETUAL"],
              api_override=['v1.realtime'])


def get_mexc_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  parameters = []
  parameters.append(dict(group_name='mexc_spot',
                         machines=machines,
                         market_type='Spot',
                         exchanges=['Mexc'],
                         symbols=["BTC-USDT", "ETH-USDT", "XRP-USDT"],
                         api_override=['v3.realtime']))
  return parameters


def get_crypto_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.us-east-1.aws',
  ]
  return dict(group_name='crypto_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Crypto'],
              symbols=["BTC-USD.PERPETUAL", "ETH-USD.PERPETUAL",
                       "BTC-USD.MONTH", "ETH-USD.QUARTER"],
              api_override=['v1.realtime'])


def get_crypto_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
      'feed-02.us-east-1.aws',
  ]
  return dict(group_name='crypto_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Crypto'],
              symbols=["BTC-USDT", "ETH-USDT", "ETH-BTC"],
              api_override=['v2.realtime'])

def get_bitget_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitget_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Bitget'],
              symbols=["BTC-USDT", "ETH-USDT", "BTC-USDC", "ETH-BTC"],
              api_override=['v1.realtime'])


def get_bitget_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bitget_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Bitget'],
              symbols=["BTC-USDT.PERPETUAL", "ETH-USDT.PERPETUAL",
                       "BTC-USD.MONTH", "ETH-USDC.QUARTER"],
              api_override=['v1.realtime'])

def get_apollox_futures_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  symbols = [
      'BTC-USDT.PERPETUAL', 'ETH-USDT.PERPETUAL',
  ]
  return dict(group_name='apollox_futures',
              machines=machines,
              market_type='Futures',
              exchanges=['Apollox'],
              symbols=symbols,
              api_override=['v1.l1_realtime', 'v1.ealtime'])

def get_mercado_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='mercado_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Mercado'],
              symbols=["BTC-BRL", "ETH-BRL", "SHIB-BRL"],
              api_override=['v4.realtime'])

def get_bit_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='bit_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Bit'],
              symbols=["BTC-USDT", "ETH-USDT", "BTC-USD", "ETH-BTC"],
              api_override=['v1.realtime'])

def get_maicoin_spot_group_parameter():
  machines = [
      'feed-05.ap-northeast-1.aws',
  ]
  return dict(group_name='maicoin_spot',
              machines=machines,
              market_type='Spot',
              exchanges=['Maicoin'],
              symbols=["BTC-USDT", "ETH-BTC", "BTC-TWD"],
              api_override=['v2.realtime'])

def generate_slurm_script(start_date,
                          end_date,
                          venv,
                          summary_mailto,
                          enabled_groups,
                          job_submit_period=None,
                          outdir_prefix=None,
                          user=None,
                          grid=None,
                          flow_root_dir=None,
                          tag=None):
  job_submit_period = 0.01 if job_submit_period is None else job_submit_period
  user = getpass.getuser()
  grid = grid or 'iosg'
  outdir_prefix = outdir_prefix or '/remote/iosg/strat-1/buckets/analysis.derived.coin/live/plot/feed_plot'  # noqa: E501

  out_dir = '%s/%s' % (outdir_prefix, start_date)

  start_date = datetime.datetime.strptime(start_date, '%Y%m%d')
  end_date = datetime.datetime.strptime(end_date, '%Y%m%d')
  trading_dates = [td for td in iterate_date(start_date.date(), end_date.date())]
  trading_dates = [td.strftime('%Y%m%d') for td in trading_dates]

  plot_parameters = []

  if 'kr' in enabled_groups:
    plot_parameters.append(get_kr_group_parameter())

  if 'okex_futures' in enabled_groups:
    plot_parameters.append(get_okex_futures_group_parameter("v3"))
    plot_parameters.append(get_okex_futures_group_parameter("v5"))

  if 'okex_spot' in enabled_groups:
    plot_parameters.append(get_okex_spot_group_parameter("v3"))
    plot_parameters.append(get_okex_spot_group_parameter("v5"))

  if 'okex_futures_swap' in enabled_groups:
    plot_parameters.append(get_okex_futures_swap_group_parameter("v3"))
    plot_parameters.append(get_okex_futures_swap_group_parameter("v5"))

  if 'huobi_futures' in enabled_groups:
    plot_parameters.append(get_huobi_futures_group_parameter())

  if 'huobi_futures_swap' in enabled_groups:
    plot_parameters.append(get_huobi_futures_swap_group_parameter())

  if 'huobi_futures_linear_swap' in enabled_groups:
    plot_parameters.append(get_huobi_futures_linear_swap_group_parameter())

  if 'huobi_spot' in enabled_groups:
    plot_parameters.append(get_huobi_spot_group_parameter())

  if 'binance_futures' in enabled_groups:
    plot_parameters.append(get_binance_futures_group_parameter())

  if 'binance_spot' in enabled_groups:
    plot_parameters.append(get_binance_spot_group_parameter())

  if 'binanceus_spot' in enabled_groups:
    plot_parameters.append(get_binanceus_spot_group_parameter())

  if 'bitmex' in enabled_groups:
    plot_parameters.append(get_bitmex_group_parameter())

  if 'bitflyer' in enabled_groups:
    plot_parameters.append(get_bitflyer_group_parameter())

  if 'bitbank' in enabled_groups:
    plot_parameters.append(get_bitbank_group_parameter())

  if 'bitfinex' in enabled_groups:
    plot_parameters.append(get_bitfinex_group_parameter())

  if 'gdax' in enabled_groups:
    plot_parameters.append(get_gdax_group_parameter())

  if 'kraken' in enabled_groups:
    plot_parameters.append(get_kraken_group_parameter())

  if 'upbit' in enabled_groups:
    plot_parameters.append(get_upbit_group_parameter())

  if 'coinone' in enabled_groups:
    plot_parameters.append(get_coinone_group_parameter())

  if 'quoinex' in enabled_groups:
    plot_parameters.append(get_quoinex_group_parameter())

  if 'bithumb' in enabled_groups:
    plot_parameters.append(get_bithumb_group_parameter())

  if 'gopax' in enabled_groups:
    plot_parameters.append(get_gopax_group_parameter())

  if 'bybit' in enabled_groups:
    plot_parameters.extend(get_bybit_spot_group_parameter())

  if 'bybit_futures' in enabled_groups:
    plot_parameters.extend(get_bybit_futures_group_parameter())

  if 'bybit-linear' in enabled_groups:
    plot_parameters.extend(get_bybit_linear_group_parameter())

  if 'ftx' in enabled_groups:
    plot_parameters.append(get_ftx_group_parameter())

  if 'ftx_futures' in enabled_groups:
    plot_parameters.append(get_ftx_futures_group_parameter())

  if 'bitstamp_spot' in enabled_groups:
    plot_parameters.append(get_bitstamp_spot_group_parameter())

  if 'korbit_spot' in enabled_groups:
    plot_parameters.append(get_korbit_spot_group_parameter())

  if 'bitbank_spot' in enabled_groups:
    plot_parameters.append(get_bitbank_spot_group_parameter())

  if 'prex_futures' in enabled_groups:
    plot_parameters.append(get_prex_futures_group_parameter())

  if 'lmaxdigital_spot' in enabled_groups:
    plot_parameters.append(get_lmaxdigital_spot_group_parameter())

  if 'deribit_futures' in enabled_groups:
    plot_parameters.append(get_deribit_futures_group_parameter())

  if 'deribit_options' in enabled_groups:
    plot_parameters.append(get_deribit_options_group_parameter(trading_dates[0]))

  if 'coincheck_spot' in enabled_groups:
    plot_parameters.append(get_coincheck_spot_group_parameter())

  if 'uniswap_spot' in enabled_groups:
    plot_parameters.append(get_uniswap_spot_group_parameter())

  if 'uniswapv3_arbitrum_spot' in enabled_groups:
    plot_parameters.append(get_uniswapv3_arbitrum_spot_group_parameter())

  if 'uniswapv2_spot' in enabled_groups:
    plot_parameters.append(get_uniswapv2_spot_group_parameter())

  if 'pancakeswap_spot' in enabled_groups:
    plot_parameters.append(get_pancakeswap_spot_group_parameter())

  if 'gateio_spot' in enabled_groups:
    plot_parameters.append(get_gateio_spot_group_parameter())

  if 'gateio_futures' in enabled_groups:
    plot_parameters.extend(get_gateio_futures_group_parameter())

  if 'phemex_spot' in enabled_groups:
    plot_parameters.append(get_phemex_spot_group_parameter())

  if 'phemex_futures' in enabled_groups:
    plot_parameters.extend(get_phemex_futures_group_parameter())

  if 'kucoin_spot' in enabled_groups:
    plot_parameters.append(get_kucoin_spot_group_parameter())

  if 'kucoin_futures' in enabled_groups:
    plot_parameters.append(get_kucoin_futures_group_parameter())
    
  if 'dydx_futures' in enabled_groups:
    plot_parameters.append(get_dydx_futures_group_parameter())

  if 'btcturk_spot' in enabled_groups:
    plot_parameters.append(get_btcturk_spot_group_parameter())

  if 'mexc_spot' in enabled_groups:
    plot_parameters.extend(get_mexc_spot_group_parameter())

  if 'mexc_futures' in enabled_groups:
    plot_parameters.append(get_mexc_futures_group_parameter())

  if 'crypto_spot' in enabled_groups:
    plot_parameters.append(get_crypto_spot_group_parameter())

  if 'crypto_futures' in enabled_groups:
    plot_parameters.append(get_crypto_futures_group_parameter())
  
  if 'bitget_spot' in enabled_groups:
    plot_parameters.append(get_bitget_spot_group_parameter())

  if 'bitget_futures' in enabled_groups:
    plot_parameters.append(get_bitget_futures_group_parameter())
  if 'apollox_futures' in enabled_groups:
    plot_parameters.append(get_apollox_futures_group_parameter())

  if 'mercado_spot' in enabled_groups:
    plot_parameters.append(get_mercado_spot_group_parameter())
  
  if 'bit_spot' in enabled_groups:
    plot_parameters.append(get_bit_spot_group_parameter())

  if 'maicoin_spot' in enabled_groups:
    plot_parameters.append(get_maicoin_spot_group_parameter())

  if tag:
    mail_subject = "[coin-feed-plot] ${SLURM_JOB_ID} summary - %s" % tag
  else:
    mail_subject = "[coin-feed-plot] ${SLURM_JOB_ID} summary"

  template_loader = jinja2.FileSystemLoader(searchpath="./")
  template_env = jinja2.Environment(loader=template_loader)

  template = template_env.get_template('coin/support/feed_tool/feed_plot_slurm/slurm_script.tmpl')
  return template.render(user=user,
                         venv=venv,
                         out_dir=out_dir,
                         job_submit_period=job_submit_period,
                         trading_dates=trading_dates,
                         parameter_map=plot_parameters,
                         mail_subject=mail_subject,
                         summary_mailto=summary_mailto)


class SBatchError(Exception):
  pass


def run_sbatch(script_filepath, sbatch_args=None, verbose=False):
  sbatch_args = sbatch_args or []
  if isinstance(sbatch_args, str):
    sbatch_args = [token for token in sbatch_args.split(' ') if token]

  sbatch_args = ['--parsable'] + sbatch_args
  if sbatch_args:
    logging.debug('sbatch_args: %s', ' '.join(sbatch_args))

  completed = subprocess.run(['sbatch', *sbatch_args, script_filepath], stdout=subprocess.PIPE)
  if completed.returncode != 0:
    raise SBatchError(completed)

  sbatch_output = completed.stdout.decode('utf-8')
  job_id = int(sbatch_output.strip())
  if verbose:
    sys.stderr.write('Job ID: %d\n' % job_id)
  return job_id


def run_feed_plot_on_slurm(start_date,
                           end_date,
                           enabled_groups,
                           job_submit_period=None,
                           outdir_prefix=None,
                           user=None,
                           grid=None,
                           venv=None,
                           force_use_venv=True,
                           summary_mailto='{user}@prestolabs.io',
                           tag=None,
                           flow_root_dir=None,
                           dry_run=False,
                           sbatch_args=None,
                           verbose=False):
  if force_use_venv and venv is None:
    user = user or getpass.getuser()
    grid = grid or 'iosg'
    venv = os.environ.get('COIN_SLURM_VENV',
                          '/remote/{grid}/home/{user}/venv'.format(grid=grid, user=user))

  if summary_mailto is not None:
    if '{user}' in summary_mailto:
      user = 'leon'
      summary_mailto = summary_mailto.format(user=user)
    if verbose:
      sys.stderr.write('Feed plot result mail will be sent to %s.\n' % summary_mailto)

  try:
    script = generate_slurm_script(start_date,
                                   end_date,
                                   venv,
                                   summary_mailto,
                                   enabled_groups,
                                   job_submit_period=job_submit_period,
                                   outdir_prefix=outdir_prefix,
                                   user=user,
                                   grid=grid,
                                   tag=tag,
                                   flow_root_dir=flow_root_dir)
    print(script)
  except NoJobError:
    if verbose:
      sys.stderr.write('No job to run.\n')
    raise

  if dry_run:
    if verbose:
      print(script)
    return

  with tempfile.NamedTemporaryFile(mode='w') as script_file:
    script_file.write(script)
    script_file.flush()

    try:
      job_id = run_sbatch(script_file.name, sbatch_args, verbose=verbose)
      if verbose:
        print(job_id)
    except SBatchError:
      if verbose:
        sys.stderr.write('Error while running sbatch. Abort.\n')
      raise

  return job_id


def main(argv):
  FLAGS = flags.FLAGS

  venv = FLAGS.venv or os.environ.get('COIN_SLURM_VENV', None)
  enabled_groups = FLAGS.enabled_groups.split(",")

  try:
    run_feed_plot_on_slurm(start_date=FLAGS.start_date,
                           end_date=FLAGS.end_date,
                           enabled_groups=enabled_groups,
                           outdir_prefix=FLAGS.outdir,
                           grid=FLAGS.grid,
                           venv=venv,
                           force_use_venv=False,
                           summary_mailto=FLAGS.mailto,
                           tag=FLAGS.tag,
                           flow_root_dir=FLAGS.flow_root_dir,
                           dry_run=FLAGS.script_only,
                           sbatch_args=FLAGS.sbatch_args,
                           verbose=True)
  except NoJobError:
    return 0
  except SBatchError:
    return 1


if __name__ == '__main__':
  logging.basicConfig(
      level='DEBUG',
      format='%(levelname)8s %(asctime)s %(name)s %(filename)s:%(lineno)d] %(message)s')
  logging.getLogger('coin.exchange.util.feed_checker').setLevel(logging.WARNING)

  flags.DEFINE_string('start_date', None, 'yyyymmdd')
  flags.DEFINE_string('end_date', None, 'yyyymmdd. Feed at end_date is not checked.')
  flags.DEFINE_string('outdir',
                      None,
                      'Output directory. Default is /remote/<grid>/home/<user>/scratch')
  flags.DEFINE_string('grid', 'iosg', 'Specify slurm grid you will use.')
  flags.DEFINE_string(
      'venv',
      None,
      'If specified, use the given virtual env. '
      'User can use venv by setting COIN_SLURM_VENV environment variable.')
  flags.DEFINE_string('mailto',
                      '{user}@prestolabs.io',
                      'If specified, it will send a mail after sim finishes.')
  flags.DEFINE_string('tag', None, 'If specified, tag will be in a result mail subject')
  flags.DEFINE_bool('script_only',
                    False,
                    'If true, it will print sbatch script and not run sbatch.')
  flags.DEFINE_string('flow_root_dir',
                      None,
                      'If specified, use the specified directory as flow roor directory.')
  flags.DEFINE_string('sbatch_args', '-n 20', 'sbatch arguments')
  flags.DEFINE_string('enabled_groups',
                      'hk,kr,okex_futures,bitmex,bitflyer,bitfinex,bitbank',
                      'Enabled groups to be plot.')
  app.run(main)
